/**
 * 爬取数据
 * (百度百科爬取地址：https://baike.baidu.com/search?word=%E7%8E%AB%E7%91%B0&pn=0&rn=0&enc=utf8)
 * 爬取条件，根据关键字模糊爬取，或者根据门、纲、科、属爬取
 * */
// const HCCrawler = require('headless-chrome-crawler');
const puppeteer = require('puppeteer');
const {
    express,
    db_mysql,
    errorData,
    strLen,
    Util,
    downLoadImg,
    host,
    port
} = require('./common-req.js')
const router = express.Router();
const http = require('http');
const https = require('https')
const cheerio = require('cheerio');
var sysPath = require('path');
// 待抓取队列
let reptileQueue = [];

router.get('/', (req, res) => {
    const name = req.query.search;
    // const categoryId = req.params.categoryId;
    // let htmlStr = `https://baike.baidu.com/search?word=${name}&pn=0&rn=0&enc=utf8`;
    let htmlStr = `https://baike.baidu.com/item/${name}`;
    var resD = res;
    let params = {
        keyWord: name,
        url: htmlStr,
        cb: (result) => {
            resD.send(result)
        }
    }
    puppeteerHandler(params)
    // resD.send('222')
    
})

// router.get('/start', (req, res) => {
//     let dda = null;
//     let ddaSql = 'select olCNName from outline order by outlineId';
//     db_mysql.select(ddaSql, null, results => {
//         if(results && results.length > 0) {
//             results.forEach((item, index) => {
//                 // 看看为何亲求的时候参数都是一个样子
//                 console.log(item.olCNName, `http://127.0.0.1:3000/plantknow/reptile?search=${item.olCNName}`)
//                 http.get(`http://127.0.0.1:3000/plantknow/reptile?search=${item.olCNName}`, function(req, res) {
//                     // console.log(req)
//                 })
//             })
//         }
//     })
//     res.send('抓取')
// })


/**
 * 获取下一个兄弟
 * 获取到的如果是para的class就是目标内容
 * */
function getNextTag($, item) {
    let className = $(item).attr('class')
    if(className.indexOf('para') > -1) {
        var text = getNextTag($, $(item).next()[0]);
        return $(item).text() + text;
    }
    return ''
}
/**
 * obj = {
     title: '栽培',
     content: [
         {
             title: 'xxx',
             content: 'saxxx'
         }
     ]
 }
 * */
function getNextTagList($, item, resArr) {
    let className = $(item).attr('class')
    if(className.indexOf('level-2') > -1) { // 递归终止条件
        return;
    }
    let obj = {
        title: '',
        content: ''
    }
    let regText = /\[\d\]/g;
    if(className.indexOf('level-3') > -1) {
        resArr.push({
            name: 'title',
            value: $(item).text().replace(regText, '').replace(/\n/g, '')
        }); // 二级标题
    } else if(className.indexOf('para') > -1){
        resArr.push({
            name: 'content',
            value: $(item).text().replace(regText, '').replace(/\n/g, '')
        }); // 内容
    }
    getNextTagList($, $(item).next()[0], resArr);
}

/**
 * 处理百度百科得到的植物信息页面
 * */
async function plantInfoHandle($, plantName, cb) {
     let regText = /\[\d\]/g;
    // 基础信息
    let baseContent = $('.lemma-summary .para:eq(0)').text();
    let baseInfo = {
        title: '基础信息',
        content: baseContent
    }
    // console.log('基础信息', baseInfo)
    // 形态特征
    let formInfo = '';
    // 生长环境
    let growInfo = '';
    // 栽培技术
    let plantMethodsInfo = '';
    // 植物信息对象
    let botanyIfo = {
        id: strLen(),
        doorId: '',
        outlineId: '',
        sectionId: '',
        genusId: '',
        name: plantName,
        baseInfo: '',
        growInfo: '',
        formInfo: '',
        cultiveInfo: '',
        src: '',
        time: new Date()
    }
    botanyIfo.baseInfo = JSON.stringify(baseInfo);
    let plantMethods = [];
    $('.level-2').each((index, item) => {
        if($(item).text().indexOf('形态') > -1) {
            // console.log('****************************************************')
            formInfo = {
                title: '形态特征',
                content: getNextTag($, $(item).next()[0]).replace(regText, '').replace(/\n/g, '')
            }
            botanyIfo.formInfo = JSON.stringify(formInfo);
            // console.log('形态', formInfo)
            // console.log('----------------------------------------------------')
            return;
        }
        if($(item).text().indexOf('生长') > -1) {
            // console.log('****************************************************')
            growInfo = {
                title: '生长环境',
                content: getNextTag($, $(item).next()[0]).replace(regText, '').replace(/\n/g, '')
            }
            botanyIfo.growInfo = JSON.stringify(growInfo);
            // console.log('生长', growInfo)
            // console.log('----------------------------------------------------')
            return;
        }
        if($(item).text().indexOf('栽培') > -1) {
            // console.log('****************************************************')
            plantMethods.push({
                name: 'title',
                value: $(item).text().replace(regText, '').replace(/\n/g, '')
            });
            getNextTagList($, $(item).next()[0], plantMethods)
            let plantContent = [];
            for(var k = 0; k<plantMethods.length; k++) {
                if(plantMethods[k].name === 'title') {
                    plantMethods[k].value = plantMethods[k].value.replace(plantName, '')
                    plantMethods[k].value = plantMethods[k].value.replace('编辑', '')
                    let obj = {
                        title: plantMethods[k].value,
                        content: ''
                    }
                    for(var j = k+1; j<plantMethods.length; j++) {
                        if(plantMethods[j].name === 'content') {
                            obj.content += plantMethods[j].value;
                        } else {
                            k = j - 1;
                            break;
                        }
                    }
                    plantContent.push(obj)
                }
            }
            botanyIfo.cultiveInfo = JSON.stringify(plantContent);
            // console.log('栽培', plantMethods, plantMethods.length)
            // console.log('----------------------------------------------------')
            return;
        }
        
    });
    // console.log(botanyIfo)
    // 获取图片
    botanyIfo.src = $('.side-content .summary-pic img').attr('src')
    let imgUrl = botanyIfo.src;
    var hostIndex = imgUrl.indexOf('/', 10);
    var fileIndex = imgUrl.lastIndexOf('pic/');
    var fileIndexLast = imgUrl.lastIndexOf('?') > -1 ? imgUrl.lastIndexOf('?') : imgUrl.length - 1;
    var hostName = imgUrl.substring(0, hostIndex + 1);
    var fileName = imgUrl.substring(fileIndex + 4, fileIndexLast)
    var path = sysPath.join(__dirname, `../public/images/${fileName}.jpg`)
    var params = {
        url: imgUrl,
        hostName: hostName,
        fileName: fileName,
        path: path
    }
    // 读取图片
    await downLoadImg(params);
    botanyIfo.src = `/public/images/${fileName}.jpg`;
    cb && cb(botanyIfo)
    let kind = []
    // 分类信息
    $('.basic-info').find('.basicInfo-item.name').each((index, item) => {
        var text = $(item).text();
        kind[index] = {
            name : text.trim(),
            value: ''
        }
    });
    $('.basic-info').find('.basicInfo-item.value').each((index, item) => {
        var text = $(item).text();
        kind[index].value = text.trim();
    });
    // 门
    let door = ''
    // 纲
    let outline = ''
    // 科
    let section = ''
    // 属
    let genus = ''
    let la = '';
    let reg = /[^\w]*/g;
    let reg1 = /[^\u4e00-\u9fa5]*/g; // 去除英语
    let reg2 = /(?:\（.*\）| \(.*\))*/g // 去除（中文）
    kind.forEach((item, index) => {
        if(item.name == '门' && item.name != '亚门') {
            door = {
                id: strLen(),
                name: item.value.replace(reg2, '').replace(reg1, '').trim()
            }
        }
        if(item.name == '纲' && item.name != '亚纲') {
            outline = {
                id: strLen(),
                rId: door.id,
                name: item.value.replace(reg2, '').replace(reg1, '').trim()
            }
        }
        if(item.name == '科' && item.name != '亚科') {
            section = {
                id: strLen(),
                rId: outline.id,
                name: item.value.replace(reg2, '').replace(reg1, '').trim()
            }
        }
        if(item.name == '属' && item.name != '亚属') {
            genus = {
                id: strLen(),
                rId: section.id,
                name: item.value.replace(reg2, '').replace(reg1, '').trim()
            }
        }
    });
    botanyIfo.doorId = await insertDoor(door);
    outline.rId = botanyIfo.doorId;
    botanyIfo.outlineId = await insertOutline(outline);
    section.rId = botanyIfo.outlineId;
    botanyIfo.sectionId = await insertSection(section);
    genus.rId = botanyIfo.sectionId;
    botanyIfo.genusId = await insertGenus(genus);
    
    insertPlant(botanyIfo)
}
/**
 * 插入门
 * @param {Object} item 插入的数据对象
 * */
async function insertDoor(item) {
    if(!item) {
        return;
    }
    let selectSql = 'select * from doors where dCNName=?'
    let insertSql =  'insert into doors(doorId, dCNName) values(?, ?)';
    let insertArr = [item.id, item.name];
    let result = await db_mysql.asyncSelect(selectSql, item.name)
    if(result && result.length > 0) {
        return result[0].doorId;
    } else {
        let results = await db_mysql.asyncInsert(insertSql, insertArr)
        console.log('插入新的门')
        return item.id
    }
}
//插入纲
async function insertOutline(item) {
    if(!item) {
        return;
    }
    let selectSql = 'select * from outline where olCNName=?'
    let insertSql =  'insert into outline(outlineId, doorId, olCNName) values(?, ?, ?)';
    let insertArr = [item.id, item.rId, item.name];
    let result = await db_mysql.asyncSelect(selectSql, item.name)
    if(result && result.length > 0) {
        return result[0].outlineId;
    } else {
        await db_mysql.asyncInsert(insertSql, insertArr)
        console.log('插入新的纲')
        return item.id
    }
}
//插入科
async function insertSection(item) {
    if(!item) {
        return;
    }
    let selectSql = 'select * from sections where secCNName=?'
    let insertSql =  'insert into sections(sectionId, outlineId, secCNName) values(?, ?, ?)';
    let insertArr = [item.id, item.rId, item.name];
    let result = await db_mysql.asyncSelect(selectSql, item.name)
    if(result && result.length > 0) {
        return result[0].sectionId;
    } else {
        await db_mysql.asyncInsert(insertSql, insertArr)
        console.log('插入新的科')
        return item.id
    }
}
// 插入属
async function insertGenus(item) {
    if(!item) {
        return;
    }
    let selectSql = 'select * from genus where genusCNName=?'
    let insertSql =  'insert into genus(genusId, sectionId, genusCNName) values(?, ?, ?)';
    let insertArr = [item.id, item.rId, item.name];
    let result = await db_mysql.asyncSelect(selectSql, item.name)
    if(result && result.length > 0) {
        return result[0].genusId;
    } else {
        await db_mysql.asyncInsert(insertSql, insertArr)
        console.log('插入新的属')
        return item.id
    }
}
/**
 * 插入信息植物信息
 * @param {Object} item 植物信息
 * */
function insertPlant(item) {
    if(!item) {
        return;
    }
    let selectSql = 'select * from botanys where name=?'
    let insertSql =  'insert into botanys(botanyId, doorId, outlineId, sectionId, genusId, name, baseInfo, growInfo, formInfo, cultivateInfo, src, time) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
    let insertArr = [item.id, item.doorId, item.outlineId, item.sectionId, item.genusId, item.name, item.baseInfo, item.growInfo, item.formInfo, item.cultiveInfo, item.src, item.time];
    let updateSql = 'update botanys set doorId=?, outlineId=?, sectionId=?, genusId=?, name=?, baseInfo=?, growInfo=?, formInfo=?, cultivateInfo=?, src=?, time=? where botanyId=?';
    let updateArr = [item.doorId, item.outlineId, item.sectionId, item.genusId, item.name, item.baseInfo, item.growInfo, item.formInfo, item.cultiveInfo, item.src, item.time];
    db_mysql.select(selectSql, item.name, result => {
        if(result && result.length > 0) {
            updateArr.push(result[0].botanyId)
            let d = new Date(result[0].time);
            let DD = d.getTime() / 1000;
            let now = new Date();
            let dd = now.getTime() / 1000;
            if(dd - DD > 1) { //2592000
                db_mysql.update(updateSql, updateArr, updateRes => {
                    if(!!updateRes && updateRes.changedRows === 1) {
                        console.log('更新植物信息成功')
                    }
                })
            }
            return;
        } else {
            db_mysql.insert(insertSql, insertArr, results => {
                console.log('插入新的植物信息')
            })
        }
    })
}



async function puppeteerHandler(item) {
    const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox']});
    // if (!browser.isConnected()) {
    //     console.log('浏览器未启动')
    //     return
    // }
    const page = await browser.newPage();
  
    // await page.goto('https://baike.baidu.com/search?word=%E7%8E%AB%E7%91%B0%E8%8A%B1&pn=0&rn=0&enc=utf8');
    await page.goto('https://baike.baidu.com/item/' + encodeURI(item.keyWord));
    const resultsSelector = '.main-content';
      // await page.waitForSelector(resultsSelector);
    const links = await page.evaluate( resultsSelector => {
        return document.querySelectorAll(resultsSelector)[0]
      }, resultsSelector);
      const content = await page.content()
    await browser.close();
    handlerResult(content, item)
}
function handlerResult(html, params) {
    const $ = cheerio.load(html, {
        decodeEntities: false
    })
    if(!$('.lemma-summary').html()) {
        cb && cb({
            data: null,
            status: 0,
            msg: '不存在该信息'
        })
        return
    }
    plantInfoHandle($, params.keyWord, result => {
        params.cb && params.cb(result)
    })
}
module.exports = router;

// 爬虫思路:
// 0. 爬取开始
// 1. 根据关键词拼接,并将拼接的路径做为爬取的起始URL
// 2. 百度百科查询,然后获取符合条件的链接,加入待爬取队列
// 3. 遍历待爬取队列,从队列中取出URL,爬取网址对应的内容
// 4. 过滤内容,将处理好的数据拼接成有用的返回
// 5. 爬取完成